package test

import cn.edu.hfut.dmic.webcollector.model.{CrawlDatums, Page}
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler
import config.{Global, MyFile}

object GetRef extends MyFile {

  class Crawler(crawlPath: String, autoParse: Boolean) extends BreadthCrawler(crawlPath, autoParse) {
    println("test")

    override def visit(page: Page, next: CrawlDatums): Unit = {
      try {
        val isSearch = !page.select(".search-results").isEmpty
        if (isSearch) {
          val list = page.select(".docsum-title").first()
          val href = list.attr("href")

              println(href,page.url())
        } else {
          val date = page.selectText(".cit")
          val doi = page.selectText(".citation-doi")
          val fullName = page.selectText(".full-name")
          val href = page.selectText(".current-id")
             println("/" + href + "/",page.url())
        }


      } catch {
        case e: Exception =>

        //  println(page.url())
      }

    }
  }



  def getPubmedId = {
    class Crawler(crawlPath: String, autoParse: Boolean) extends BreadthCrawler(crawlPath, autoParse) {
      println("test")

      override def visit(page: Page, next: CrawlDatums): Unit = {
        try {
          val isSearch = !page.select(".search-results").isEmpty
          if (isSearch) {
            val list = page.select(".docsum-title").first()
            val href = list.attr("href")

            //    println(href)
          } else {
            val date = page.selectText(".cit")
            val doi = page.selectText(".citation-doi")
            val fullName = page.selectText(".full-name")
            val href = page.selectText(".current-id")
            //   println("/" + href + "/")
          }


        } catch {
          case e: Exception =>

            println(page.url())
        }

      }
    }
  }

  def get = {
    class Crawler(crawlPath: String, autoParse: Boolean) extends BreadthCrawler(crawlPath, autoParse) {

      override def visit(page: Page, next: CrawlDatums): Unit = {
        try {

          val date = page.selectText(".cit").split(";").head
          val doi = page.selectText(".citation-doi")
          val fullName = page.selectText(".full-name")
          val href = page.selectText(".current-id")
          val title = page.selectText(".heading-title")
          val abstracts = page.selectText(".abstract-content")

          //        println(Seq(title,date,fullName,doi,href,abstracts).mkString("\t"))

          println(page.url())
        } catch {
          case e: Exception =>

          //   println(page.url())
        }

      }
    }
  }

  def main(args: Array[String]): Unit = {
    val code = new Crawler("I:\\hordeum_db\\tmp", false)
    code.setThreads(1)

    s"${Global.path}/ref".listFile.foreach { x =>
      val name  = x.getName.dropRight(4).replaceAll(" ","%20")
      code.addSeed("https://pubmed.ncbi.nlm.nih.gov/?term=" + name)
    }
/*    s"${Global.path}/refHref.txt".readLines.foreach { x =>
      code.addSeed("https://pubmed.ncbi.nlm.nih.gov" + x)
    }*/
    code.start(1)

  }


}
